#!/usr/bin/env python
# -*- coding: utf8 -*-

"""
parser.py

Parse webpage and return informations required by Spider.

THIS IS PART OF PYSPI,
DO NOT USE DIRECTLY
"""

import re

class Parser(object):
    """
    provides useful methods to get informations about webpage
    """
    def __init__(self):
        pass
    
    def getInfo(self, page):
        """
        get all required informations about page
        :parameters:
            page : string
                webpage
        :return:
            (pageTitle, pageType, pageSize, [pageEmails]) : tuple
                informations about webpage
        """
        return self.__getTitle(page), self.__getType(page), \
                self.__getSize(page), self.__getMails(page)
    
    def __getType(self, page):
        """
        get type of delivered webpage
        :parameters:
            page : string
                delivered webpage
        :return:
            doctype : string
                type of page or 'UNKNOWN' if not described
        """
        if re.match('<\\!DOCTYPE HTML>', page, re.I):
            doctype = 'HTML 5'
        else:
            try:
                doctype = re.findall('(X?HTML \\d\\.\\d{1,2})',\
                        page, re.I)[0]
            except IndexError:
                doctype = 'UNKNOWN'
        return re.sub('\s', '', doctype)

    @staticmethod
    def getUrls(page):
        """
        get urls from page
        :parameters:
            page : string
                delivered webpage
        :return:
            urls : list
                list of urls founded in page
        """
        results = re.findall("""(?#Protocol)(?:(?:ht|f)tp(?:s?)\:\/\/|~
                    \/|\/)?(?#Username:Password)(?:\w+:\w+@)?(?#Subdo
                    mains)(?:(?:[-\w]+\.)+(?#TopLevel Domains)(?:com|
                    org|net|gov|mil|biz|info|mobi|name|aero|jobs|muse
                    um|travel|[a-z]{2}))(?#Port)(?::[\d]{1,5})?(?#Dir
                    ectories)(?:(?:(?:\/(?:[-\w~!$+|.,=]|%[a-f\d]{2})
                    +)+|\/)+|\?|#)?(?#Query)(?:(?:\?(?:[-\w~!$+|.,*:]
                    |%[a-f\d{2}])+=?(?:[-\w~!$+|.,*:=]|%[a-f\d]{2})*)
                    (?:&(?:[-\w~!$+|.,*:]|%[a-f\d{2}])+=?(?:[-\w~!$+|
                    .,*:=]|%[a-f\d]{2})*)*)*(?#Anchor)(?:#(?:[-\w~!$+
                    |.,*:=]|%[a-f\d]{2})*)?""", page)
        return {}.fromkeys(results).keys() #uniqify
    def __getMails(self, page):
        """
        get mails from page
        :parameters:
            page : string
                delivered webpage
        :return:
            mails : list
                list of mails founded in page
        """
        mails = [mail[0] for mail in\
                re.findall("([\w\-\.]+@(\w[\w\-]+\.)+[\w\-]+)",page)]
        return {}.fromkeys(mails).keys() #uniqify
    
    def __getTitle(self, page):
        """
        get title from page
        :parameters:
            page : string
                delivered webpage
        :return:
            title : string
                title founded in page
        """
        result = re.findall(u'<title>.*</title>', page)
        if result:
            return re.sub('\s', '_', re.split('[<>]',\
                            result[0])[2])
        else:
            return 'unknown'

    def __getSize(self, page):
        """
        get size of page
        :parameters:
            page : string
                delivered webpage
        :return:
            size : string
                size of page in string
        """
        return str(len(page))

if __name__ == "__main__":
    print "Parser is a part of pyspi project.\
        \nDon't use it directly"

